#-*-coding:utf-8-*- 
__author__ = 'Ares'
#将豆瓣电影的250排名的影片抓取下来保存在excel中


from bs4 import BeautifulSoup
import re
import requests
from openpyxl import Workbook
from openpyxl import load_workbook

def download_page(url):
    heards={'User-Agent':"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"}
    movieUrl=requests.get(url,headers=heards)

    soup=BeautifulSoup(movieUrl.text,'lxml')
    movie_list=soup.find('ol',class_='grid_view').find_all('li')

    name=[]         #电影名字
    star_con=[]     #评价人数
    level_list=[]        #评分
    info_list=[]    #短评
    id_list=[]      #序号

    year_list=[]    #年份
    country_list=[]
    mtype_list=[]

    #print movie_list
    for movie in movie_list:
        ID=movie.find('div',class_='pic').find('em').get_text()
        #print ID
        #电影名字
        movie_name=movie.find('div',class_='hd').find('span',class_='title').get_text()
        #评分
        level_star=movie.find('div',class_='star').find('span',class_='rating_num').get_text()
        #评论人数
        star_num=movie.find('div',class_='star').find_all('span')[3].get_text()

        #年份，国家，电影类型
        message=movie.find('div',class_='bd').find('p').get_text().strip()
        year=message.split("\n")[1]
        year=year.split('/')

        myear=year[0].strip()+u"年"
        country=year[1].strip()
        mtype=year[2].strip()
        #print year
        # for y in year:
        #     print y.strip()

        #短评
        #先获取标签再判断标签有没有值
        info=movie.find('div',class_='bd').find_all('span',class_='inq')
        #info = movie.find('div', class_='bd').find_all('span', class_='inq')[0].get_text()
        if info:    #判断标签有没有值
            info=info[0].get_text()
        else:
            info="无"

        #print ID,movie_name
        id_list.append(ID)
        info_list.append(info)
        name.append(movie_name)
        star_con.append(star_num)
        level_list.append(level_star)

        year_list.append(myear)
        country_list.append(country)
        mtype_list.append(mtype)

    return  id_list,info_list,name,star_con,level_list,year_list,country_list,mtype_list

def main():
    fileName=u"豆瓣电影top250.xlsx"
    wb=Workbook()
    ws=wb.active
    ws.title=u"豆瓣电影top250"

    ws.cell('A1').value = "序号"
    ws.cell('B1').value = "名字"
    ws.cell('C1').value = "上映年份"
    ws.cell('D1').value = "国家"
    ws.cell('E1').value = "类型"
    ws.cell('F1').value = "评论人数"
    ws.cell('G1').value = "评分"
    ws.cell('H1').value = "短评"

    id_list=[]
    info_list=[]
    name_list=[]
    star_list=[]
    num_list=[]
    level_list=[]

    year_list=[]
    country_list=[]
    mtype_list=[]
    url="https://movie.douban.com/top250"
    for i in range(0,10):
        page_value="?start="+str(i*25)+"&filter="   #i*25
        #?start=225&filter=
        page_url=url+page_value
        id_list,info_list,name_list,num_list,level_list,year_list,country_list,mtype_list=download_page(page_url)

        for (Id,info,name,num,level,myear,country,mtype) in zip(id_list,info_list,name_list,num_list,level_list,year_list,country_list,mtype_list):
            col_A="A"+str(int(Id)+1)
            col_B="B"+str(int(Id)+1)
            col_C="C"+str(int(Id)+1)
            col_D="D"+str(int(Id)+1)
            col_E="E"+str(int(Id)+1)
            col_F = "F" + str(int(Id) + 1)
            col_G = "G" + str(int(Id) + 1)
            col_H = "H" + str(int(Id) + 1)


            ws.cell(col_A).value = str(Id)
            ws.cell(col_B).value = name
            ws.cell(col_C).value = myear
            ws.cell(col_D).value = country#str(level)
            ws.cell(col_E).value = mtype
            ws.cell(col_F).value = num
            ws.cell(col_G).value = str(level)
            ws.cell(col_H).value = info


    wb.save(filename=fileName)


if __name__=="__main__":
    print r"开始抓取[https://movie.douban.com/top250]中的电影列表............"
    main()
    print r"抓取电影top250完成......."